<!DOCTYPE html>
<html lang="zh-cn">
  <head>
    
    <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="generator" content="Hugo 0.56.3 with theme Tranquilpeak 0.4.7-BETA">
<meta name="author" content="iFence">
<meta name="keywords" content="">
<meta name="description" content="map与flatmap的区别 初学scala的map和flatmap算子的时候，有些搞不清楚他们的区别。后来学了python感觉稍微明白了一些。 先看一段代码： val conf = new SparkConf().setMaster(&quot;local&quot;).setAppName(&quot;nums&quot;) val sc = new SparkContext(conf) val list1 = List(&quot;hello hello world python python is number&quot;,&quot;I am gosing no code no code&quot;) val string = sc.parallelize(list1).flatMap(_.split(&quot; &quot;)).map(_ &#43; &quot; &quot;).foreach(print(_)) println() sc.parallelize(list1).map(_.split(&quot; &quot;)).map(_.toList).foreach(print(_)) /** 输出结果： hello hello world python python is number I am gosing no code no code List(hello, hello, world, python, python,">


<meta property="og:description" content="map与flatmap的区别 初学scala的map和flatmap算子的时候，有些搞不清楚他们的区别。后来学了python感觉稍微明白了一些。 先看一段代码： val conf = new SparkConf().setMaster(&quot;local&quot;).setAppName(&quot;nums&quot;) val sc = new SparkContext(conf) val list1 = List(&quot;hello hello world python python is number&quot;,&quot;I am gosing no code no code&quot;) val string = sc.parallelize(list1).flatMap(_.split(&quot; &quot;)).map(_ &#43; &quot; &quot;).foreach(print(_)) println() sc.parallelize(list1).map(_.split(&quot; &quot;)).map(_.toList).foreach(print(_)) /** 输出结果： hello hello world python python is number I am gosing no code no code List(hello, hello, world, python, python,">
<meta property="og:type" content="article">
<meta property="og:title" content="Map和Flatmap的区别">
<meta name="twitter:title" content="Map和Flatmap的区别">
<meta property="og:url" content="https://ifence.gitee.io/2019/08/map%E5%92%8Cflatmap%E7%9A%84%E5%8C%BA%E5%88%AB/">
<meta property="twitter:url" content="https://ifence.gitee.io/2019/08/map%E5%92%8Cflatmap%E7%9A%84%E5%8C%BA%E5%88%AB/">
<meta property="og:site_name" content="iFence&#39;s Nest">
<meta property="og:description" content="map与flatmap的区别 初学scala的map和flatmap算子的时候，有些搞不清楚他们的区别。后来学了python感觉稍微明白了一些。 先看一段代码： val conf = new SparkConf().setMaster(&quot;local&quot;).setAppName(&quot;nums&quot;) val sc = new SparkContext(conf) val list1 = List(&quot;hello hello world python python is number&quot;,&quot;I am gosing no code no code&quot;) val string = sc.parallelize(list1).flatMap(_.split(&quot; &quot;)).map(_ &#43; &quot; &quot;).foreach(print(_)) println() sc.parallelize(list1).map(_.split(&quot; &quot;)).map(_.toList).foreach(print(_)) /** 输出结果： hello hello world python python is number I am gosing no code no code List(hello, hello, world, python, python,">
<meta name="twitter:description" content="map与flatmap的区别 初学scala的map和flatmap算子的时候，有些搞不清楚他们的区别。后来学了python感觉稍微明白了一些。 先看一段代码： val conf = new SparkConf().setMaster(&quot;local&quot;).setAppName(&quot;nums&quot;) val sc = new SparkContext(conf) val list1 = List(&quot;hello hello world python python is number&quot;,&quot;I am gosing no code no code&quot;) val string = sc.parallelize(list1).flatMap(_.split(&quot; &quot;)).map(_ &#43; &quot; &quot;).foreach(print(_)) println() sc.parallelize(list1).map(_.split(&quot; &quot;)).map(_.toList).foreach(print(_)) /** 输出结果： hello hello world python python is number I am gosing no code no code List(hello, hello, world, python, python,">
<meta property="og:locale" content="zh-cn">

  
    <meta property="article:published_time" content="2019-08-04T16:59:46">
  
  
    <meta property="article:modified_time" content="2019-08-04T16:59:46">
  
  
  
    
      <meta property="article:section" content="Spark">
    
  
  
    
      <meta property="article:tag" content="map&amp;flatMap">
    
      <meta property="article:tag" content="Spark">
    
  


<meta name="twitter:card" content="summary">











  <meta property="og:image" content="https://avatars1.githubusercontent.com/u/53120100?s=460&v=4">
  <meta property="twitter:image" content="https://avatars1.githubusercontent.com/u/53120100?s=460&v=4">


    <title>Map和Flatmap的区别</title>

    <link rel="icon" href="https://ifence.gitee.io/favicon.png">
    

    

    <link rel="canonical" href="https://ifence.gitee.io/2019/08/map%E5%92%8Cflatmap%E7%9A%84%E5%8C%BA%E5%88%AB/">

    
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" integrity="sha256-eZrrJcwDc/3uDhsdt61sL2oOBY362qM3lon1gyExkL0=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.4/jquery.fancybox.min.css" integrity="sha256-vuXZ9LGmmwtjqFX1F+EKin1ThZMub58gKULUyf0qECk=" crossorigin="anonymous" />
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.4/helpers/jquery.fancybox-thumbs.min.css" integrity="sha256-SEa4XYAHihTcEP1f5gARTB2K26Uk8PsndQYHQC1f4jU=" crossorigin="anonymous" />
    
    
    <link rel="stylesheet" href="https://ifence.gitee.io/css/style-twzjdbqhmnnacqs0pwwdzcdbt8yhv8giawvjqjmyfoqnvazl0dalmnhdkvp7.min.css" />
    
    

    
      
<script type="application/javascript">
var doNotTrack = false;
if (!doNotTrack) {
	(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
	(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
	m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
	})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
	ga('create', 'UA-123-45', 'auto');
	
	ga('send', 'pageview');
}
</script>

    
    
  </head>

  <body>
    <div id="blog">
      <header id="header" data-behavior="5">
  <i id="btn-open-sidebar" class="fa fa-lg fa-bars"></i>
  <div class="header-title">
    <a class="header-title-link" href="https://ifence.gitee.io/">iFence&#39;s Nest</a>
  </div>
  
    
      <a class="header-right-picture "
         href="https://ifence.gitee.io/#about">
    
    
    
      
        <img class="header-picture" src="https://avatars1.githubusercontent.com/u/53120100?s=460&amp;v=4" alt="作者的图片" />
      
    
    </a>
  
</header>

      <nav id="sidebar" data-behavior="5">
  <div class="sidebar-container">
    
      <div class="sidebar-profile">
        <a href="https://ifence.gitee.io/#about">
          <img class="sidebar-profile-picture" src="https://avatars1.githubusercontent.com/u/53120100?s=460&amp;v=4" alt="作者的图片" />
        </a>
        <h4 class="sidebar-profile-name">iFence</h4>
        
          <h5 class="sidebar-profile-bio">虽千万人吾往矣</h5>
        
      </div>
    
    <ul class="sidebar-buttons">
      
  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://ifence.gitee.io/">
    
      <i class="sidebar-button-icon fa fa-lg fa-home"></i>
      
      <span class="sidebar-button-desc">首页</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://ifence.gitee.io/archives">
    
      <i class="sidebar-button-icon fa fa-lg fa-archive"></i>
      
      <span class="sidebar-button-desc">归档</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://ifence.gitee.io/tags">
    
      <i class="sidebar-button-icon fa fa-lg fa-tags"></i>
      
      <span class="sidebar-button-desc">标签</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://ifence.gitee.io/categories">
    
      <i class="sidebar-button-icon fa fa-lg fa-bookmark"></i>
      
      <span class="sidebar-button-desc">分类</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://ifence.gitee.io/about">
    
      <i class="sidebar-button-icon fa fa-lg fa-user"></i>
      
      <span class="sidebar-button-desc">关于</span>
    </a>
  </li>


    </ul>
    <ul class="sidebar-buttons">
      
  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://github.com/iFence" target="_blank" rel="noopener">
    
      <i class="sidebar-button-icon fa fa-lg fa-github"></i>
      
      <span class="sidebar-button-desc">GitHub</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://blog.csdn.net/Vector97" target="_blank" rel="noopener">
    
      <i class="sidebar-button-icon fa fa-lg fa-codiepie"></i>
      
      <span class="sidebar-button-desc">CSDN</span>
    </a>
  </li>

  <li class="sidebar-button">
    
      <a class="sidebar-button-link " href="https://www.jianshu.com/u/424596d51c5a" target="_blank" rel="noopener">
    
      <i class="sidebar-button-icon fa fa-lg fa-gratipay"></i>
      
      <span class="sidebar-button-desc">简书</span>
    </a>
  </li>


    </ul>
    <ul class="sidebar-buttons">
      

    </ul>
  </div>
</nav>

      

      <div id="main" data-behavior="5"
        class="
               hasCoverMetaIn
               ">
        <article class="post" itemscope itemType="http://schema.org/BlogPosting">
          
          
            <div class="post-header main-content-wrap text-left">
  
    <h1 class="post-title" itemprop="headline">
      Map和Flatmap的区别
    </h1>
  
  
  <div class="postShorten-meta post-meta">
    
      <time itemprop="datePublished" datetime="2019-08-04T16:59:46&#43;08:00">
        
  八月 4, 2019

      </time>
    
    
  
  
    <span>发布在</span>
    
      <a class="category-link" href="https://ifence.gitee.io/categories/spark">Spark</a>
    
  

  </div>

</div>
          
          <div class="post-content markdown" itemprop="articleBody">
            <div class="main-content-wrap">
              

<h3 id="map与flatmap的区别">map与flatmap的区别</h3>

<p>初学scala的map和flatmap算子的时候，有些搞不清楚他们的区别。后来学了python感觉稍微明白了一些。</p>

<p>先看一段代码：</p>

<pre><code class="language-java">val conf = new SparkConf().setMaster(&quot;local&quot;).setAppName(&quot;nums&quot;)
val sc = new SparkContext(conf)
val list1 = List(&quot;hello hello world python python is number&quot;,&quot;I am gosing no code no code&quot;)
val string = sc.parallelize(list1).flatMap(_.split(&quot; &quot;)).map(_ + &quot; &quot;).foreach(print(_))
println()
sc.parallelize(list1).map(_.split(&quot; &quot;)).map(_.toList).foreach(print(_))
/**
输出结果：
hello hello world python python is number I am gosing no code no code 
List(hello, hello, world, python, python, is, number)List(I, am, gosing, no, code, no, code)
**/
</code></pre>

<p>为了更直观看一下python版本：</p>

<pre><code class="language-python">def map_flatmap():
    sc = SparkContext(&quot;local&quot;, &quot;wordcount app&quot;)
    list1 = [&quot;hello hello world&quot;, &quot;no code no code&quot;]
    list2 = [1, 2, 3, 4, 5, 6]
    list3 = [[1, 2, 3], [4, 5, 6]]
    list1_rdd = sc.parallelize(list1)
    print(&quot;str map&quot;, end=&quot;: &quot;)
    list1_rdd.map(lambda line: line.split(&quot; &quot;)).foreach(lambda word_list: print(word_list, end=&quot;\t&quot;))
    print()
    print(&quot;str flatmap&quot;, end=&quot;: &quot;)
    list1_rdd.flatMap(lambda line: line.split(&quot; &quot;)).foreach(lambda word: print(word, end=&quot;\t&quot;))
    print()
    print(&quot;num map&quot;, end=&quot;: &quot;)
    list2_rdd = sc.parallelize(list2)
    list2_rdd.map(lambda n: n * 2).foreach(lambda n: print(n, end=&quot;\t&quot;))
    print()
    print(&quot;num flatmap&quot;, end=&quot;: &quot;)
    list3_rdd = sc.parallelize(list3)
    list3_rdd.flatMap(lambda ln: [i ** 3 for i in ln]).foreach(lambda n: print(n, end=&quot;\t&quot;))

map_flatmap()
'''
输出结果：
str map: ['hello', 'hello', 'world']	['no', 'code', 'no', 'code']	
str flatmap: hello	hello	world	no	code	no	code	
num map: 2	4	6	8	10	12	
num flatmap: 1	8	27	64	125	216		
'''
</code></pre>

<p>map是对给定的集合进行遍历（上面的例子就是对list1进行遍历），对集合的每个元素进行处理（上面的例子就是对str进行分割），最终结果放回到一个集合中去（上面的例子就是放回到了list中），所以最终的结果外层list放了两个list子元素。map遍历出来的一个个元素是集合，处理以后的元素依然是集合，如果是简单类型，则处理结束依然是简单类型</p>

<p>flatmap是对给定的集合进行遍历，然后对遍历出的每一个元素进行处理，处理结果不再放入一个集合，而是直接放到外面。flatmap最开始的处理与map一致，只是它接受的对象遍历出来的元素必须依然是可以迭代的，他会把这个可迭代的子元素迭代出来，这个过程是flat，最终形成一个大集合，再进行map操作。</p>

<p>所以flatmap更像是mapflat</p>

<pre><code class="language-java">//map与flatmap的又一个例子
val ln = List(1,2,3)
val numRdd = sc.parallelize(ln)
numRdd.map(_.to(3)).foreach(println(_))
numRdd.flatMap(_.to(3)).foreach(println(_))
/*
输出结果：
Range(1, 2, 3)
Range(2, 3)
Range(3)
1	2	3	2	3	3
*/
</code></pre>

<p>上面可以看到，map与flatmap可接受集合的子元素可以是简单类型，不需要必须得可迭代。但是传入的参数处理以后，flatmap，要求结果是可以遍历的,flatmap要求：<code>T =&gt; TraversableOnce</code></p>

              
            </div>
          </div>
          <div id="post-footer" class="post-footer main-content-wrap">
            
              
                
                
                  <div class="post-footer-tags">
                    <span class="text-color-light text-small">标签</span><br/>
                    
  <a class="tag tag--primary tag--small" href="https://ifence.gitee.io/tags/mapflatmap/">map&amp;flatMap</a>

  <a class="tag tag--primary tag--small" href="https://ifence.gitee.io/tags/spark/">Spark</a>

                  </div>
                
              
            
            <div class="post-actions-wrap">
  
      <nav >
        <ul class="post-actions post-action-nav">
          
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://ifence.gitee.io/2019/08/log4j%E7%9A%84api%E4%BD%BF%E7%94%A8%E8%AF%A6%E8%A7%A3/" data-tooltip="Log4j的api使用详解">
              
                  <i class="fa fa-angle-left"></i>
                  <span class="hide-xs hide-sm text-small icon-ml">下一篇</span>
                </a>
            </li>
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://ifence.gitee.io/2019/08/%E4%BD%BF%E7%94%A8wsl%E8%BF%9B%E8%A1%8C%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%BC%80%E5%8F%91/" data-tooltip="使用WSL进行大数据开发">
              
                  <span class="hide-xs hide-sm text-small icon-mr">上一篇</span>
                  <i class="fa fa-angle-right"></i>
                </a>
            </li>
          
        </ul>
      </nav>
    <ul class="post-actions post-action-share" >
      
        <li class="post-action hide-lg hide-md hide-sm">
          <a class="post-action-btn btn btn--default btn-open-shareoptions" href="#btn-open-shareoptions">
            <i class="fa fa-share-alt"></i>
          </a>
        </li>
        
      
      
        <li class="post-action">
          <a class="post-action-btn btn btn--default" href="#disqus_thread">
            <i class="fa fa-comment-o"></i>
          </a>
        </li>
      
      <li class="post-action">
        
          <a class="post-action-btn btn btn--default" href="#">
        
          <i class="fa fa-list"></i>
        </a>
      </li>
    </ul>
  
</div>

            
              
                <div id="disqus_thread">
  <noscript>Please enable JavaScript to view the <a href="//disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
</div>
              
            
          </div>
        </article>
        <footer id="footer" class="main-content-wrap">
  <span class="copyrights">
    &copy; 2019 iFence. All Rights Reserved
  </span>
</footer>

      </div>
      <div id="bottom-bar" class="post-bottom-bar" data-behavior="5">
        <div class="post-actions-wrap">
  
      <nav >
        <ul class="post-actions post-action-nav">
          
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://ifence.gitee.io/2019/08/log4j%E7%9A%84api%E4%BD%BF%E7%94%A8%E8%AF%A6%E8%A7%A3/" data-tooltip="Log4j的api使用详解">
              
                  <i class="fa fa-angle-left"></i>
                  <span class="hide-xs hide-sm text-small icon-ml">下一篇</span>
                </a>
            </li>
            <li class="post-action">
              
                <a class="post-action-btn btn btn--default tooltip--top" href="https://ifence.gitee.io/2019/08/%E4%BD%BF%E7%94%A8wsl%E8%BF%9B%E8%A1%8C%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%BC%80%E5%8F%91/" data-tooltip="使用WSL进行大数据开发">
              
                  <span class="hide-xs hide-sm text-small icon-mr">上一篇</span>
                  <i class="fa fa-angle-right"></i>
                </a>
            </li>
          
        </ul>
      </nav>
    <ul class="post-actions post-action-share" >
      
        <li class="post-action hide-lg hide-md hide-sm">
          <a class="post-action-btn btn btn--default btn-open-shareoptions" href="#btn-open-shareoptions">
            <i class="fa fa-share-alt"></i>
          </a>
        </li>
        
      
      
        <li class="post-action">
          <a class="post-action-btn btn btn--default" href="#disqus_thread">
            <i class="fa fa-comment-o"></i>
          </a>
        </li>
      
      <li class="post-action">
        
          <a class="post-action-btn btn btn--default" href="#">
        
          <i class="fa fa-list"></i>
        </a>
      </li>
    </ul>
  
</div>

      </div>
      <div id="share-options-bar" class="share-options-bar" data-behavior="5">
  <i id="btn-close-shareoptions" class="fa fa-close"></i>
  <ul class="share-options">
    
  </ul>
</div>
<div id="share-options-mask" class="share-options-mask"></div>
    </div>
    
    <div id="about">
  <div id="about-card">
    <div id="about-btn-close">
      <i class="fa fa-remove"></i>
    </div>
    
      <img id="about-card-picture" src="https://avatars1.githubusercontent.com/u/53120100?s=460&amp;v=4" alt="作者的图片" />
    
    <h4 id="about-card-name">iFence</h4>
    
      <div id="about-card-bio">虽千万人吾往矣</div>
    
    
      <div id="about-card-job">
        <i class="fa fa-briefcase"></i>
        <br/>
        大数据开发
      </div>
    
    
      <div id="about-card-location">
        <i class="fa fa-map-marker"></i>
        <br/>
        中国 山东
      </div>
    
  </div>
</div>

    

    
  
    
      <div id="cover" style="background-image:url('https://ifence.gitee.io/images/cover.jpg');"></div>
    
  


    
<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>

  <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js" integrity="sha256-/BfiIkHlHoVihZdc6TFuj7MmJ0TWcWsMXkeDFwhi0zw=" crossorigin="anonymous"></script>

<script src="https://cdnjs.cloudflare.com/ajax/libs/fancybox/2.1.7/js/jquery.fancybox.min.js" integrity="sha256-GEAnjcTqVP+vBp3SSc8bEDQqvWAZMiHyUSIorrWwH50=" crossorigin="anonymous"></script>


<script src="https://ifence.gitee.io/js/script-pcw6v3xilnxydl1vddzazdverrnn9ctynvnxgwho987mfyqkuylcb1nlt.min.js"></script>


<script lang="javascript">
window.onload = updateMinWidth;
window.onresize = updateMinWidth;
document.getElementById("sidebar").addEventListener("transitionend", updateMinWidth);
function updateMinWidth() {
  var sidebar = document.getElementById("sidebar");
  var main = document.getElementById("main");
  main.style.minWidth = "";
  var w1 = getComputedStyle(main).getPropertyValue("min-width");
  var w2 = getComputedStyle(sidebar).getPropertyValue("width");
  var w3 = getComputedStyle(sidebar).getPropertyValue("left");
  main.style.minWidth = `calc(${w1} - ${w2} - ${w3})`;
}
</script>

<script>
$(document).ready(function() {
  hljs.configure({ classPrefix: '', useBR: false });
  $('pre.code-highlight > code, pre > code').each(function(i, block) {
    if (!$(this).hasClass('codeblock')) {
      $(this).addClass('codeblock');
    }
    hljs.highlightBlock(block);
  });
});
</script>


  
    
      <script>
        var disqus_config = function () {
          this.page.url = 'https:\/\/ifence.gitee.io\/2019\/08\/map%E5%92%8Cflatmap%E7%9A%84%E5%8C%BA%E5%88%AB\/';
          
            this.page.identifier = '\/2019\/08\/map%E5%92%8Cflatmap%E7%9A%84%E5%8C%BA%E5%88%AB\/'
          
        };
        (function() {
          
          
          if (window.location.hostname == "localhost") {
            return;
          }
          var d = document, s = d.createElement('script');
          var disqus_shortname = 'hugo-tranquilpeak-theme';
          s.src = '//' + disqus_shortname + '.disqus.com/embed.js';

          s.setAttribute('data-timestamp', +new Date());
          (d.head || d.body).appendChild(s);
        })();
      </script>
    
  




    
  </body>
</html>

